Benchmark#

Whenever we make changes to the code we will rerun a simple benchmark in order to keep track of how the simulation results might change over time. It is in general hard to verify that the equations are implemented correctly because there exist no analytic solutions to the equations. However, by running the same benchmark over time we can see if the changes we make will impact the simulation results. Here the different biomarkers are taken from traces evaluated a the center of at slab that is simulated for 1000 ms (see the benchmark in the library for more details)

from pathlib import Path
import json
import matplotlib.pyplot as plt
import numpy as np
from collections import defaultdict
import pandas as pd
import plotly.graph_objects as go
from plotly.subplots import make_subplots

pd.options.display.max_columns = 5

benchmark_folder = Path(".").absolute().parent / "benchmarks"

all_hashes = [f.name for f in benchmark_folder.iterdir()]

print(all_hashes)

['5d5bd73', '10592e9', '71a7b0e', '12ad5f2', 'd5245d4', '9b246fe', '9d5c1b1', '3398fd2', 'c9b3528', 'e616f0a', 'f5511ff', 'f73b25e', '1ed82dc', '8b11e8d', 'a3a23b2', 'de8a7ff', '159c6bc', 'c7df02b', 'de72206', 'ac17a41', '38ad52c', 'fbfcf55', '3ed0bc0', '09a7e5b', '739dc1c', '695cd7b', '32e43ff', 'e99e183', '0648726', '47edd23', '93c349e', 'df9ad53', '37fb7d1', '5992724', 'cbc3038', '7d2252c', '3b1d61b', 'dbbec69', 'a4ea024', 'edc973c', '813ba9c', '1353445', '5580bef', '0221cc5', '4d3a653', '6e55904', '68926df', 'bc47c96', '37f4c61', 'c242e33', '0e0091e', '7f51a50', '42363b0', 'f63afc9', '5edda13', '1f7912f', '90b65d3', 'f702955', '9d72060', '60e69df', '97e958d', '0d80766', '69503d0', 'd44db58', 'e3254b0', '322191d', 'e8e6d35', '4cc3e7a', '80a11fc', '6dde0c0', 'e36932a', '9f2fd38', '67aba10', 'cb33e06', '172470c', 'f38605c', '5cc1a44', 'cc84512', '999fcfa', '0c195ff', '947b107', 'f214f80', '5a86d28', '7ffac90', '68ae7bb', '376167c', '630ed09', '5282d4f', 'cf7cc9d', '2fe1309', '3a87bf1', 'ce433ea']

You can checkout a specific commit by going to the link

ComputationalPhysiology/simcardems

where you swap out GIT_COMMIT_HASH with the specific hash, i.e

for hash in all_hashes:
    print(f"https://github.com/ComputationalPhysiology/simcardems/commit/{hash}")

https://github.com/ComputationalPhysiology/simcardems/commit/5d5bd73
https://github.com/ComputationalPhysiology/simcardems/commit/10592e9
https://github.com/ComputationalPhysiology/simcardems/commit/71a7b0e
https://github.com/ComputationalPhysiology/simcardems/commit/12ad5f2
https://github.com/ComputationalPhysiology/simcardems/commit/d5245d4
https://github.com/ComputationalPhysiology/simcardems/commit/9b246fe
https://github.com/ComputationalPhysiology/simcardems/commit/9d5c1b1
https://github.com/ComputationalPhysiology/simcardems/commit/3398fd2
https://github.com/ComputationalPhysiology/simcardems/commit/c9b3528
https://github.com/ComputationalPhysiology/simcardems/commit/e616f0a
https://github.com/ComputationalPhysiology/simcardems/commit/f5511ff
https://github.com/ComputationalPhysiology/simcardems/commit/f73b25e
https://github.com/ComputationalPhysiology/simcardems/commit/1ed82dc
https://github.com/ComputationalPhysiology/simcardems/commit/8b11e8d
https://github.com/ComputationalPhysiology/simcardems/commit/a3a23b2
https://github.com/ComputationalPhysiology/simcardems/commit/de8a7ff
https://github.com/ComputationalPhysiology/simcardems/commit/159c6bc
https://github.com/ComputationalPhysiology/simcardems/commit/c7df02b
https://github.com/ComputationalPhysiology/simcardems/commit/de72206
https://github.com/ComputationalPhysiology/simcardems/commit/ac17a41
https://github.com/ComputationalPhysiology/simcardems/commit/38ad52c
https://github.com/ComputationalPhysiology/simcardems/commit/fbfcf55
https://github.com/ComputationalPhysiology/simcardems/commit/3ed0bc0
https://github.com/ComputationalPhysiology/simcardems/commit/09a7e5b
https://github.com/ComputationalPhysiology/simcardems/commit/739dc1c
https://github.com/ComputationalPhysiology/simcardems/commit/695cd7b
https://github.com/ComputationalPhysiology/simcardems/commit/32e43ff
https://github.com/ComputationalPhysiology/simcardems/commit/e99e183
https://github.com/ComputationalPhysiology/simcardems/commit/0648726
https://github.com/ComputationalPhysiology/simcardems/commit/47edd23
https://github.com/ComputationalPhysiology/simcardems/commit/93c349e
https://github.com/ComputationalPhysiology/simcardems/commit/df9ad53
https://github.com/ComputationalPhysiology/simcardems/commit/37fb7d1
https://github.com/ComputationalPhysiology/simcardems/commit/5992724
https://github.com/ComputationalPhysiology/simcardems/commit/cbc3038
https://github.com/ComputationalPhysiology/simcardems/commit/7d2252c
https://github.com/ComputationalPhysiology/simcardems/commit/3b1d61b
https://github.com/ComputationalPhysiology/simcardems/commit/dbbec69
https://github.com/ComputationalPhysiology/simcardems/commit/a4ea024
https://github.com/ComputationalPhysiology/simcardems/commit/edc973c
https://github.com/ComputationalPhysiology/simcardems/commit/813ba9c
https://github.com/ComputationalPhysiology/simcardems/commit/1353445
https://github.com/ComputationalPhysiology/simcardems/commit/5580bef
https://github.com/ComputationalPhysiology/simcardems/commit/0221cc5
https://github.com/ComputationalPhysiology/simcardems/commit/4d3a653
https://github.com/ComputationalPhysiology/simcardems/commit/6e55904
https://github.com/ComputationalPhysiology/simcardems/commit/68926df
https://github.com/ComputationalPhysiology/simcardems/commit/bc47c96
https://github.com/ComputationalPhysiology/simcardems/commit/37f4c61
https://github.com/ComputationalPhysiology/simcardems/commit/c242e33
https://github.com/ComputationalPhysiology/simcardems/commit/0e0091e
https://github.com/ComputationalPhysiology/simcardems/commit/7f51a50
https://github.com/ComputationalPhysiology/simcardems/commit/42363b0
https://github.com/ComputationalPhysiology/simcardems/commit/f63afc9
https://github.com/ComputationalPhysiology/simcardems/commit/5edda13
https://github.com/ComputationalPhysiology/simcardems/commit/1f7912f
https://github.com/ComputationalPhysiology/simcardems/commit/90b65d3
https://github.com/ComputationalPhysiology/simcardems/commit/f702955
https://github.com/ComputationalPhysiology/simcardems/commit/9d72060
https://github.com/ComputationalPhysiology/simcardems/commit/60e69df
https://github.com/ComputationalPhysiology/simcardems/commit/97e958d
https://github.com/ComputationalPhysiology/simcardems/commit/0d80766
https://github.com/ComputationalPhysiology/simcardems/commit/69503d0
https://github.com/ComputationalPhysiology/simcardems/commit/d44db58
https://github.com/ComputationalPhysiology/simcardems/commit/e3254b0
https://github.com/ComputationalPhysiology/simcardems/commit/322191d
https://github.com/ComputationalPhysiology/simcardems/commit/e8e6d35
https://github.com/ComputationalPhysiology/simcardems/commit/4cc3e7a
https://github.com/ComputationalPhysiology/simcardems/commit/80a11fc
https://github.com/ComputationalPhysiology/simcardems/commit/6dde0c0
https://github.com/ComputationalPhysiology/simcardems/commit/e36932a
https://github.com/ComputationalPhysiology/simcardems/commit/9f2fd38
https://github.com/ComputationalPhysiology/simcardems/commit/67aba10
https://github.com/ComputationalPhysiology/simcardems/commit/cb33e06
https://github.com/ComputationalPhysiology/simcardems/commit/172470c
https://github.com/ComputationalPhysiology/simcardems/commit/f38605c
https://github.com/ComputationalPhysiology/simcardems/commit/5cc1a44
https://github.com/ComputationalPhysiology/simcardems/commit/cc84512
https://github.com/ComputationalPhysiology/simcardems/commit/999fcfa
https://github.com/ComputationalPhysiology/simcardems/commit/0c195ff
https://github.com/ComputationalPhysiology/simcardems/commit/947b107
https://github.com/ComputationalPhysiology/simcardems/commit/f214f80
https://github.com/ComputationalPhysiology/simcardems/commit/5a86d28
https://github.com/ComputationalPhysiology/simcardems/commit/7ffac90
https://github.com/ComputationalPhysiology/simcardems/commit/68ae7bb
https://github.com/ComputationalPhysiology/simcardems/commit/376167c
https://github.com/ComputationalPhysiology/simcardems/commit/630ed09
https://github.com/ComputationalPhysiology/simcardems/commit/5282d4f
https://github.com/ComputationalPhysiology/simcardems/commit/cf7cc9d
https://github.com/ComputationalPhysiology/simcardems/commit/2fe1309
https://github.com/ComputationalPhysiology/simcardems/commit/3a87bf1
https://github.com/ComputationalPhysiology/simcardems/commit/ce433ea

data = {}
for git_hash in all_hashes:
    results_file = benchmark_folder / git_hash / "results.json"
    if not results_file.is_file():
        print(f"Results does not exist for version {git_hash}")
        continue
    value = json.loads(results_file.read_text())
    if "mesh" in value.keys():
        # We have the wrong file
        continue
    data[git_hash] = value
        

results = defaultdict(list)

for git_hash, d in data.items():
    results["git_hash"].append(git_hash)
    for key, value in d.items():
        results[key].append(value)

df = pd.DataFrame(results)
df["timestamp"] = pd.to_datetime(df["timestamp"])
df
git_hash import_time ... rel_max_displacement_perc_z time_to_max_displacement_z
0 5d5bd73 9.010000e-07 ... 0.486886 0.05
1 10592e9 9.000000e-07 ... 0.486886 0.05
2 71a7b0e 8.000000e-07 ... 0.486886 0.05
3 12ad5f2 9.000000e-07 ... 0.486909 0.05
4 d5245d4 8.000000e-07 ... 0.486886 0.05
... ... ... ... ... ...
85 5282d4f 1.200000e-06 ... 0.486886 0.05
86 cf7cc9d 9.000000e-07 ... 0.249432 138.05
87 2fe1309 9.000000e-07 ... 0.486920 0.05
88 3a87bf1 1.000000e-06 ... 0.486886 0.05
89 ce433ea 8.000000e-07 ... 0.486920 0.05

90 rows × 51 columns

df = df.sort_values(by='timestamp')
df
git_hash import_time ... rel_max_displacement_perc_z time_to_max_displacement_z
52 f63afc9 1.001000e-06 ... 0.249432 138.05
84 630ed09 1.000000e-06 ... 0.249432 138.05
43 4d3a653 1.100000e-06 ... 0.249432 138.05
77 0c195ff 1.000000e-06 ... 0.249432 138.05
17 c7df02b 9.000000e-07 ... 0.249432 138.05
... ... ... ... ... ...
58 60e69df 9.000000e-07 ... 0.486886 0.05
15 de8a7ff 9.000000e-07 ... 0.486886 0.05
85 5282d4f 1.200000e-06 ... 0.486886 0.05
54 1f7912f 9.000000e-07 ... 0.486886 0.05
39 813ba9c 1.200000e-06 ... 0.486886 0.05

90 rows × 51 columns

df[["simcardems_version", "timestamp", "git_hash"]]
simcardems_version timestamp git_hash
52 2022.3.0 2022-10-06 16:25:06.830650 f63afc9
84 2022.3.0 2022-10-06 16:55:04.711879 630ed09
43 2022.3.0 2022-10-06 20:09:07.230901 4d3a653
77 2022.3.0 2022-10-06 20:19:12.227464 0c195ff
17 2022.3.0 2022-10-06 20:28:25.015389 c7df02b
... ... ... ...
58 2023.2.0 2023-03-28 11:17:31.206004 60e69df
15 2023.2.0 2023-03-31 08:23:50.167394 de8a7ff
85 2023.2.0 2023-03-31 10:02:00.632466 5282d4f
54 2023.2.0 2023-03-31 13:39:54.344860 1f7912f
39 2023.3.0 2023-03-31 13:46:14.985304 813ba9c

90 rows × 3 columns

def get_ylim(values):
    if np.isclose(values, 0).all():
        return (-0.01, 0.01)
    y_mean = np.mean(values)
    y_max = max(np.max(values), y_mean + 0.01 * abs(y_mean))
    y_min = min(np.min(values), y_mean - 0.01 * abs(y_mean))
    return (y_min, y_max)

columns = [c for c in df.columns if c not in ["timestamp", "simcardems_version", "git_hash"]]
git_hash = df["git_hash"]
versions = df["simcardems_version"]
dates = [t.date() for t in df["timestamp"]]

text = []
for h,v, t in zip(git_hash, versions, dates):
    text.append("\n".join([
        f"<br>Git Hash: {h}</br>",
        f"<br>Version: {v}</br>",
        f"<br>Timestamp {t}</br>",
    ]))
    
fig = make_subplots(rows=len(columns), cols=1, subplot_titles=columns, shared_xaxes=True)

yranges = []
for i, c in enumerate(columns):
    
    row = i + 1
    col = 1
    
    fig.add_trace(
        go.Scatter(x=df["timestamp"], y=df[c], text=text, hovertemplate="%{text}"),
        row=row, col=col
    )
    fig.update_yaxes(range=get_ylim(df[c]),row=row, col=col)

fig.update_layout(
    height=5000,
    showlegend=False
)
fig.show()